03-validate-models
Libraries and Data
library(brms)
library(rstan)
library(tidyverse)
library(cowplot)
library(colorblindr)
library(ggpubr)
library(tidybayes)
library(modelr)
# recommended code to speed up stan
# see https://github.com/stan-dev/rstan/wiki/RStan-Getting-Started
rstan_options(auto_write = TRUE)
options(mc.cores = parallel::detectCores())
vis_display_order <- c("point","barchart","interval","table","hopsdist","dotplot","hops","density")
max_eval = 30
rq3 <- read_csv("data/rq3.csv") %>%
mutate(
eval_period_normalized = ((evalPeriod - max_eval) / max_eval) + 0.5
)rq3 %>%
ggplot(aes(x = evalPeriod, y = dv, color = treatment)) +
stat_summary(fun.data = mean_se) +
geom_hline(yintercept = 1) +
stat_smooth(method = lm) +
facet_grid(.~treatment) +
theme(legend.position = "none")
Priors
Commented out.
get_prior(bf(
dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken),
phi ~ eval_period_normalized * treatment
),
data = rq3, family = Beta)## prior class coef
## (flat) b
## (flat) b eval_period_normalized
## (flat) b eval_period_normalized:treatmentdensity
## (flat) b eval_period_normalized:treatmentdotplot
## (flat) b eval_period_normalized:treatmenthops
## (flat) b eval_period_normalized:treatmenthopsdist
## (flat) b eval_period_normalized:treatmentinterval
## (flat) b eval_period_normalized:treatmentpoint
## (flat) b eval_period_normalized:treatmenttable
## (flat) b treatmentdensity
## (flat) b treatmentdotplot
## (flat) b treatmenthops
## (flat) b treatmenthopsdist
## (flat) b treatmentinterval
## (flat) b treatmentpoint
## (flat) b treatmenttable
## student_t(3, 0, 2.5) Intercept
## student_t(3, 0, 2.5) sd
## student_t(3, 0, 2.5) sd
## student_t(3, 0, 2.5) sd Intercept
## (flat) b
## (flat) b eval_period_normalized
## (flat) b eval_period_normalized:treatmentdensity
## (flat) b eval_period_normalized:treatmentdotplot
## (flat) b eval_period_normalized:treatmenthops
## (flat) b eval_period_normalized:treatmenthopsdist
## (flat) b eval_period_normalized:treatmentinterval
## (flat) b eval_period_normalized:treatmentpoint
## (flat) b eval_period_normalized:treatmenttable
## (flat) b treatmentdensity
## (flat) b treatmentdotplot
## (flat) b treatmenthops
## (flat) b treatmenthopsdist
## (flat) b treatmentinterval
## (flat) b treatmentpoint
## (flat) b treatmenttable
## student_t(3, 0, 2.5) Intercept
## group resp dpar nlpar bound source
## default
## (vectorized)
## (vectorized)
## (vectorized)
## (vectorized)
## (vectorized)
## (vectorized)
## (vectorized)
## (vectorized)
## (vectorized)
## (vectorized)
## (vectorized)
## (vectorized)
## (vectorized)
## (vectorized)
## (vectorized)
## default
## default
## usertoken (vectorized)
## usertoken (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi (vectorized)
## phi defaultpr_beta = c(
prior(normal(0, 1), class = b),
# these prior intercepts are wide and cover 0 (50% on the logit scale), but
# also assume some likely better-than-50% performance on average --- this
# was chosen to aid convergence during the pilot, but does not have a strong
# impact on final estimates.
prior(normal(2, 2), class = Intercept),
prior(normal(2, 2), class = Intercept, dpar = phi),
prior(normal(0, 1), class = b, dpar = phi),
prior(student_t(3, 0, 1), class = sd)
)Fit model
# save model - core model
# dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken),
# phi ~ eval_period_normalized * treatment),
mbeta = brm(bf(
dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken),
phi ~ eval_period_normalized * treatment),
data = rq3,
prior = pr_beta,
control = list(adapt_delta = 0.9995, max_treedepth = 15, stepsize = 0.005),
warmup = warmup, iter = iter, thin = thin,
family = Beta
)
save(mbeta, file = "models/fit1.rda")mbeta2 = brm(bf(
dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken),
phi ~ eval_period_normalized * treatment),
data = rq3,
prior = pr_beta,
control = list(adapt_delta = 0.9995, max_treedepth = 15, stepsize = 0.005),
warmup = warmup, iter = iter, thin = thin,
family = Beta
) # phi ~ eval_period_normalized ),
# challenger: phi eval_period_nor
save(mbeta2, file = "models/fit2.rda")mbeta3 = brm(bf(
dv ~ eval_period_normalized + treatment + (1|usertoken),
phi ~ eval_period_normalized),
data = rq3,
prior = pr_beta,
control = list(adapt_delta = 0.9995, max_treedepth = 15, stepsize = 0.005),
warmup = warmup, iter = iter, thin = thin,
family = Beta
) # phi ~ eval_period_normalized ),
# challenger: phi eval_period_nor
save(mbeta3, file = "models/fit3.rda")# save model - core model
# dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken),
# phi ~ eval_period_normalized * treatment),
mbeta4 = brm(bf(
dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken),
phi ~ eval_period_normalized * treatment),
data = rq3,
#prior = pr_beta,
control = list(adapt_delta = 0.9995, max_treedepth = 15, stepsize = 0.005),
warmup = warmup, iter = iter, thin = thin,
family = Beta
)
save(mbeta4, file = "models/fit4.rda")mbeta5 = brm(bf(
dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken),
phi ~ eval_period_normalized * treatment),
data = rq3,
#prior = pr_beta,
control = list(adapt_delta = 0.9995, max_treedepth = 15, stepsize = 0.005),
warmup = warmup, iter = iter, thin = thin,
family = Beta
) # phi ~ eval_period_normalized ),
# challenger: phi eval_period_nor
save(mbeta5, file = "models/fit5.rda")mbeta6 = brm(bf(
dv ~ eval_period_normalized + treatment + (1|usertoken),
phi ~ eval_period_normalized),
data = rq3,
#prior = pr_beta,
control = list(adapt_delta = 0.9995, max_treedepth = 15, stepsize = 0.005),
warmup = warmup, iter = iter, thin = thin,
family = Beta
) # phi ~ eval_period_normalized ),
# challenger: phi eval_period_nor
save(mbeta6, file = "models/fit6.rda")# load model
load("models/fit1.rda")
load("models/fit2.rda")
load("models/fit3.rda")
load("models/fit4.rda")
load("models/fit5.rda")
load("models/fit6.rda")Model results
summary(mbeta)## Family: beta
## Links: mu = logit; phi = log
## Formula: dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1 | usertoken)
## phi ~ eval_period_normalized * treatment
## Data: rq3 (Number of observations: 1393)
## Samples: 4 chains, each with iter = 4000; warmup = 2000; thin = 2;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~usertoken (Number of levels: 199)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.63 0.04 0.55 0.72 1.00 1946 3028
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI
## Intercept 1.85 0.13 1.60
## phi_Intercept 2.62 0.11 2.40
## eval_period_normalized 1.10 0.15 0.80
## treatmentdensity -0.51 0.19 -0.89
## treatmentdotplot -0.38 0.19 -0.73
## treatmenthops -0.40 0.20 -0.80
## treatmenthopsdist -0.31 0.19 -0.68
## treatmentinterval -0.02 0.19 -0.38
## treatmentpoint 0.18 0.19 -0.19
## treatmenttable -0.27 0.19 -0.65
## eval_period_normalized:treatmentdensity -1.10 0.25 -1.60
## eval_period_normalized:treatmentdotplot -0.24 0.26 -0.73
## eval_period_normalized:treatmenthops -0.36 0.26 -0.89
## eval_period_normalized:treatmenthopsdist -0.10 0.24 -0.57
## eval_period_normalized:treatmentinterval -0.16 0.24 -0.63
## eval_period_normalized:treatmentpoint -0.08 0.26 -0.58
## eval_period_normalized:treatmenttable -0.55 0.26 -1.05
## phi_eval_period_normalized 0.19 0.26 -0.33
## phi_treatmentdensity -0.55 0.16 -0.86
## phi_treatmentdotplot -0.86 0.15 -1.14
## phi_treatmenthops -0.65 0.16 -0.97
## phi_treatmenthopsdist -0.34 0.16 -0.66
## phi_treatmentinterval -0.21 0.16 -0.52
## phi_treatmentpoint -0.62 0.16 -0.93
## phi_treatmenttable -0.69 0.16 -1.01
## phi_eval_period_normalized:treatmentdensity -0.20 0.43 -1.00
## phi_eval_period_normalized:treatmentdotplot -0.18 0.40 -0.94
## phi_eval_period_normalized:treatmenthops -0.36 0.43 -1.21
## phi_eval_period_normalized:treatmenthopsdist -0.30 0.43 -1.15
## phi_eval_period_normalized:treatmentinterval -0.02 0.41 -0.83
## phi_eval_period_normalized:treatmentpoint 0.64 0.38 -0.10
## phi_eval_period_normalized:treatmenttable -0.02 0.43 -0.88
## u-95% CI Rhat Bulk_ESS Tail_ESS
## Intercept 2.11 1.00 953 1796
## phi_Intercept 2.84 1.00 1993 3304
## eval_period_normalized 1.39 1.00 2046 2732
## treatmentdensity -0.13 1.00 1126 1841
## treatmentdotplot -0.02 1.00 1198 2203
## treatmenthops -0.01 1.00 1270 2416
## treatmenthopsdist 0.07 1.00 1248 2304
## treatmentinterval 0.34 1.00 1101 2130
## treatmentpoint 0.55 1.00 1328 2097
## treatmenttable 0.11 1.00 1390 2195
## eval_period_normalized:treatmentdensity -0.61 1.00 2139 3062
## eval_period_normalized:treatmentdotplot 0.27 1.00 2537 3092
## eval_period_normalized:treatmenthops 0.16 1.00 2644 3168
## eval_period_normalized:treatmenthopsdist 0.36 1.00 2644 2984
## eval_period_normalized:treatmentinterval 0.30 1.00 2291 3127
## eval_period_normalized:treatmentpoint 0.42 1.00 2559 3424
## eval_period_normalized:treatmenttable -0.03 1.00 2363 3181
## phi_eval_period_normalized 0.71 1.00 2094 2700
## phi_treatmentdensity -0.24 1.00 2118 2977
## phi_treatmentdotplot -0.57 1.00 2228 3085
## phi_treatmenthops -0.33 1.00 2310 2934
## phi_treatmenthopsdist -0.03 1.00 2413 3142
## phi_treatmentinterval 0.10 1.00 2330 3118
## phi_treatmentpoint -0.32 1.00 2461 2809
## phi_treatmenttable -0.38 1.00 2502 3227
## phi_eval_period_normalized:treatmentdensity 0.65 1.00 2154 3480
## phi_eval_period_normalized:treatmentdotplot 0.62 1.00 2019 3243
## phi_eval_period_normalized:treatmenthops 0.49 1.00 2457 3217
## phi_eval_period_normalized:treatmenthopsdist 0.51 1.00 2681 3386
## phi_eval_period_normalized:treatmentinterval 0.77 1.00 2801 3404
## phi_eval_period_normalized:treatmentpoint 1.39 1.00 2339 3261
## phi_eval_period_normalized:treatmenttable 0.82 1.00 2847 3386
##
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).summary(mbeta2)## Family: beta
## Links: mu = logit; phi = log
## Formula: dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1 | usertoken)
## phi ~ eval_period_normalized
## Data: rq3 (Number of observations: 1386)
## Samples: 4 chains, each with iter = 4000; warmup = 2000; thin = 2;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~usertoken (Number of levels: 198)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.63 0.04 0.55 0.72 1.00 2032 3026
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI
## Intercept 1.73 0.13 1.48 1.98
## phi_Intercept 2.09 0.04 2.01 2.17
## eval_period_normalized 0.98 0.17 0.65 1.33
## treatmentdensity -0.38 0.19 -0.75 -0.01
## treatmentdotplot -0.17 0.18 -0.53 0.18
## treatmenthops -0.26 0.20 -0.66 0.13
## treatmenthopsdist -0.24 0.19 -0.61 0.13
## treatmentinterval 0.01 0.19 -0.36 0.37
## treatmentpoint 0.31 0.18 -0.04 0.66
## treatmenttable -0.11 0.19 -0.48 0.25
## eval_period_normalized:treatmentdensity -0.97 0.25 -1.46 -0.46
## eval_period_normalized:treatmentdotplot -0.06 0.24 -0.52 0.41
## eval_period_normalized:treatmenthops -0.20 0.26 -0.70 0.32
## eval_period_normalized:treatmenthopsdist 0.01 0.25 -0.49 0.49
## eval_period_normalized:treatmentinterval -0.08 0.25 -0.58 0.42
## eval_period_normalized:treatmentpoint -0.15 0.25 -0.63 0.33
## eval_period_normalized:treatmenttable -0.44 0.25 -0.95 0.06
## phi_eval_period_normalized 0.17 0.13 -0.08 0.42
## Rhat Bulk_ESS Tail_ESS
## Intercept 1.00 1312 2368
## phi_Intercept 1.00 3180 3388
## eval_period_normalized 1.00 2202 2820
## treatmentdensity 1.00 1480 2432
## treatmentdotplot 1.00 1336 2097
## treatmenthops 1.00 1400 2491
## treatmenthopsdist 1.00 1429 2356
## treatmentinterval 1.00 1411 2573
## treatmentpoint 1.00 1485 2119
## treatmenttable 1.00 1348 2244
## eval_period_normalized:treatmentdensity 1.00 2323 3033
## eval_period_normalized:treatmentdotplot 1.00 2504 3125
## eval_period_normalized:treatmenthops 1.00 2589 3246
## eval_period_normalized:treatmenthopsdist 1.00 2557 3065
## eval_period_normalized:treatmentinterval 1.00 2311 3260
## eval_period_normalized:treatmentpoint 1.00 2465 3014
## eval_period_normalized:treatmenttable 1.00 2289 2892
## phi_eval_period_normalized 1.00 3426 3630
##
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).summary(mbeta3)## Family: beta
## Links: mu = logit; phi = log
## Formula: dv ~ eval_period_normalized + treatment + (1 | usertoken)
## phi ~ eval_period_normalized
## Data: rq3 (Number of observations: 1386)
## Samples: 4 chains, each with iter = 4000; warmup = 2000; thin = 2;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~usertoken (Number of levels: 198)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.63 0.04 0.55 0.71 1.00 2266 3120
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept 1.73 0.13 1.48 1.98 1.00 1442
## phi_Intercept 2.07 0.04 1.99 2.16 1.00 3014
## eval_period_normalized 0.74 0.07 0.60 0.88 1.00 3111
## treatmentdensity -0.37 0.20 -0.75 0.02 1.00 1485
## treatmentdotplot -0.17 0.18 -0.53 0.18 1.00 1587
## treatmenthops -0.26 0.20 -0.64 0.13 1.00 1579
## treatmenthopsdist -0.24 0.20 -0.62 0.15 1.00 1803
## treatmentinterval 0.01 0.19 -0.34 0.38 1.00 1683
## treatmentpoint 0.30 0.18 -0.06 0.65 1.00 1674
## treatmenttable -0.11 0.19 -0.50 0.25 1.00 1735
## phi_eval_period_normalized 0.16 0.13 -0.08 0.41 1.00 3377
## Tail_ESS
## Intercept 2407
## phi_Intercept 3186
## eval_period_normalized 3553
## treatmentdensity 2502
## treatmentdotplot 2747
## treatmenthops 2613
## treatmenthopsdist 2419
## treatmentinterval 2659
## treatmentpoint 2344
## treatmenttable 2375
## phi_eval_period_normalized 3520
##
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).summary(mbeta4)## Family: beta
## Links: mu = logit; phi = log
## Formula: dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1 | usertoken)
## phi ~ eval_period_normalized * treatment
## Data: rq3 (Number of observations: 1386)
## Samples: 4 chains, each with iter = 4000; warmup = 2000; thin = 2;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~usertoken (Number of levels: 198)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.63 0.04 0.56 0.72 1.00 1919 2903
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI
## Intercept 1.89 0.14 1.63
## phi_Intercept 2.69 0.11 2.46
## eval_period_normalized 1.21 0.17 0.88
## treatmentdensity -0.56 0.20 -0.96
## treatmentdotplot -0.43 0.19 -0.81
## treatmenthops -0.47 0.21 -0.88
## treatmenthopsdist -0.37 0.20 -0.77
## treatmentinterval -0.06 0.20 -0.44
## treatmentpoint 0.12 0.20 -0.27
## treatmenttable -0.32 0.20 -0.72
## eval_period_normalized:treatmentdensity -1.27 0.26 -1.79
## eval_period_normalized:treatmentdotplot -0.37 0.27 -0.90
## eval_period_normalized:treatmenthops -0.50 0.27 -1.04
## eval_period_normalized:treatmenthopsdist -0.22 0.26 -0.72
## eval_period_normalized:treatmentinterval -0.28 0.25 -0.75
## eval_period_normalized:treatmentpoint -0.18 0.28 -0.74
## eval_period_normalized:treatmenttable -0.69 0.28 -1.25
## phi_eval_period_normalized 0.27 0.38 -0.50
## phi_treatmentdensity -0.62 0.16 -0.94
## phi_treatmentdotplot -0.94 0.15 -1.23
## phi_treatmenthops -0.73 0.17 -1.07
## phi_treatmenthopsdist -0.41 0.17 -0.75
## phi_treatmentinterval -0.28 0.16 -0.60
## phi_treatmentpoint -0.70 0.16 -1.02
## phi_treatmenttable -0.76 0.16 -1.08
## phi_eval_period_normalized:treatmentdensity -0.34 0.55 -1.42
## phi_eval_period_normalized:treatmentdotplot -0.29 0.51 -1.28
## phi_eval_period_normalized:treatmenthops -0.49 0.54 -1.53
## phi_eval_period_normalized:treatmenthopsdist -0.40 0.53 -1.46
## phi_eval_period_normalized:treatmentinterval -0.10 0.52 -1.09
## phi_eval_period_normalized:treatmentpoint 0.60 0.49 -0.38
## phi_eval_period_normalized:treatmenttable -0.12 0.55 -1.17
## u-95% CI Rhat Bulk_ESS Tail_ESS
## Intercept 2.16 1.00 1120 1849
## phi_Intercept 2.90 1.00 2144 2947
## eval_period_normalized 1.54 1.00 2236 2871
## treatmentdensity -0.16 1.00 1450 2327
## treatmentdotplot -0.05 1.00 1355 2204
## treatmenthops -0.06 1.00 1545 2295
## treatmenthopsdist 0.03 1.00 1348 2015
## treatmentinterval 0.31 1.00 1295 2340
## treatmentpoint 0.52 1.00 1426 2714
## treatmenttable 0.07 1.00 1548 2533
## eval_period_normalized:treatmentdensity -0.76 1.00 2619 3341
## eval_period_normalized:treatmentdotplot 0.15 1.00 2452 3408
## eval_period_normalized:treatmenthops 0.03 1.00 2536 3355
## eval_period_normalized:treatmenthopsdist 0.27 1.00 2681 3267
## eval_period_normalized:treatmentinterval 0.23 1.00 2688 2813
## eval_period_normalized:treatmentpoint 0.37 1.00 2377 3025
## eval_period_normalized:treatmenttable -0.14 1.00 2309 2939
## phi_eval_period_normalized 0.99 1.00 1742 2647
## phi_treatmentdensity -0.31 1.00 2538 3216
## phi_treatmentdotplot -0.62 1.00 2359 2816
## phi_treatmenthops -0.42 1.00 2368 3352
## phi_treatmenthopsdist -0.08 1.00 2574 3387
## phi_treatmentinterval 0.04 1.00 2242 3193
## phi_treatmentpoint -0.39 1.00 2355 2962
## phi_treatmenttable -0.43 1.00 2284 3140
## phi_eval_period_normalized:treatmentdensity 0.75 1.00 2019 2986
## phi_eval_period_normalized:treatmentdotplot 0.73 1.00 2046 2837
## phi_eval_period_normalized:treatmenthops 0.58 1.00 2183 3213
## phi_eval_period_normalized:treatmenthopsdist 0.63 1.00 2260 3108
## phi_eval_period_normalized:treatmentinterval 0.92 1.00 2081 3156
## phi_eval_period_normalized:treatmentpoint 1.55 1.00 2305 3132
## phi_eval_period_normalized:treatmenttable 0.96 1.00 2196 2882
##
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).summary(mbeta5)## Family: beta
## Links: mu = logit; phi = log
## Formula: dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1 | usertoken)
## phi ~ eval_period_normalized * treatment
## Data: rq3 (Number of observations: 1386)
## Samples: 4 chains, each with iter = 4000; warmup = 2000; thin = 2;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~usertoken (Number of levels: 198)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.64 0.04 0.56 0.72 1.00 1903 2275
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI
## Intercept 1.90 0.13 1.64
## phi_Intercept 2.69 0.11 2.47
## eval_period_normalized 1.21 0.16 0.89
## treatmentdensity -0.56 0.20 -0.96
## treatmentdotplot -0.44 0.19 -0.82
## treatmenthops -0.47 0.21 -0.88
## treatmenthopsdist -0.38 0.20 -0.77
## treatmentinterval -0.07 0.19 -0.45
## treatmentpoint 0.11 0.20 -0.28
## treatmenttable -0.33 0.20 -0.72
## eval_period_normalized:treatmentdensity -1.27 0.26 -1.78
## eval_period_normalized:treatmentdotplot -0.37 0.27 -0.88
## eval_period_normalized:treatmenthops -0.50 0.27 -1.05
## eval_period_normalized:treatmenthopsdist -0.23 0.26 -0.75
## eval_period_normalized:treatmentinterval -0.27 0.25 -0.77
## eval_period_normalized:treatmentpoint -0.18 0.28 -0.72
## eval_period_normalized:treatmenttable -0.70 0.28 -1.25
## phi_eval_period_normalized 0.28 0.37 -0.44
## phi_treatmentdensity -0.63 0.16 -0.96
## phi_treatmentdotplot -0.94 0.15 -1.24
## phi_treatmenthops -0.74 0.17 -1.07
## phi_treatmenthopsdist -0.41 0.16 -0.74
## phi_treatmentinterval -0.29 0.16 -0.60
## phi_treatmentpoint -0.70 0.16 -1.01
## phi_treatmenttable -0.76 0.16 -1.08
## phi_eval_period_normalized:treatmentdensity -0.34 0.54 -1.39
## phi_eval_period_normalized:treatmentdotplot -0.30 0.49 -1.28
## phi_eval_period_normalized:treatmenthops -0.49 0.53 -1.53
## phi_eval_period_normalized:treatmenthopsdist -0.42 0.53 -1.47
## phi_eval_period_normalized:treatmentinterval -0.10 0.50 -1.07
## phi_eval_period_normalized:treatmentpoint 0.58 0.49 -0.38
## phi_eval_period_normalized:treatmenttable -0.13 0.53 -1.17
## u-95% CI Rhat Bulk_ESS Tail_ESS
## Intercept 2.17 1.00 1024 1619
## phi_Intercept 2.90 1.00 2211 2977
## eval_period_normalized 1.53 1.00 2064 2784
## treatmentdensity -0.17 1.00 1199 1933
## treatmentdotplot -0.08 1.00 1289 1992
## treatmenthops -0.05 1.00 1182 1793
## treatmenthopsdist 0.03 1.00 1241 1790
## treatmentinterval 0.30 1.00 1279 1891
## treatmentpoint 0.49 1.00 1071 2336
## treatmenttable 0.06 1.01 1312 2401
## eval_period_normalized:treatmentdensity -0.76 1.00 2553 3269
## eval_period_normalized:treatmentdotplot 0.15 1.00 2421 3115
## eval_period_normalized:treatmenthops 0.03 1.00 2502 3248
## eval_period_normalized:treatmenthopsdist 0.27 1.00 2391 3105
## eval_period_normalized:treatmentinterval 0.23 1.00 2598 3319
## eval_period_normalized:treatmentpoint 0.38 1.00 2505 2636
## eval_period_normalized:treatmenttable -0.15 1.00 2328 2798
## phi_eval_period_normalized 0.97 1.00 1461 2402
## phi_treatmentdensity -0.31 1.00 2389 2880
## phi_treatmentdotplot -0.64 1.00 2378 3064
## phi_treatmenthops -0.41 1.00 2443 2921
## phi_treatmenthopsdist -0.09 1.00 2300 3299
## phi_treatmentinterval 0.04 1.00 2578 3030
## phi_treatmentpoint -0.38 1.00 2450 3419
## phi_treatmenttable -0.45 1.00 2269 3127
## phi_eval_period_normalized:treatmentdensity 0.74 1.00 2134 3033
## phi_eval_period_normalized:treatmentdotplot 0.68 1.00 1713 2866
## phi_eval_period_normalized:treatmenthops 0.55 1.00 2209 3066
## phi_eval_period_normalized:treatmenthopsdist 0.59 1.00 1784 2858
## phi_eval_period_normalized:treatmentinterval 0.89 1.00 1801 2978
## phi_eval_period_normalized:treatmentpoint 1.54 1.00 1921 2841
## phi_eval_period_normalized:treatmenttable 0.91 1.00 2220 2978
##
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).summary(mbeta6)## Family: beta
## Links: mu = logit; phi = log
## Formula: dv ~ eval_period_normalized + treatment + (1 | usertoken)
## phi ~ eval_period_normalized
## Data: rq3 (Number of observations: 1386)
## Samples: 4 chains, each with iter = 4000; warmup = 2000; thin = 2;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~usertoken (Number of levels: 198)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.63 0.04 0.55 0.72 1.00 2327 2536
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept 1.74 0.14 1.46 2.02 1.00 1416
## phi_Intercept 2.07 0.04 1.99 2.15 1.00 3020
## eval_period_normalized 0.75 0.07 0.60 0.89 1.00 3047
## treatmentdensity -0.40 0.20 -0.79 0.00 1.00 1526
## treatmentdotplot -0.19 0.19 -0.56 0.18 1.00 1511
## treatmenthops -0.29 0.21 -0.69 0.13 1.00 1785
## treatmenthopsdist -0.26 0.20 -0.65 0.14 1.00 1611
## treatmentinterval 0.01 0.20 -0.38 0.40 1.00 1508
## treatmentpoint 0.30 0.20 -0.09 0.68 1.00 1539
## treatmenttable -0.13 0.20 -0.53 0.27 1.00 1548
## phi_eval_period_normalized 0.17 0.13 -0.08 0.42 1.00 3605
## Tail_ESS
## Intercept 2138
## phi_Intercept 3372
## eval_period_normalized 3208
## treatmentdensity 2584
## treatmentdotplot 2322
## treatmenthops 2503
## treatmenthopsdist 2568
## treatmentinterval 2635
## treatmentpoint 2148
## treatmenttable 2674
## phi_eval_period_normalized 3739
##
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).PPC across models
pp_check(mbeta)
pp_check(mbeta2)
pp_check(mbeta3)
pp_check(mbeta4)
pp_check(mbeta5)
pp_check(mbeta6)
Conditional Effects
conditional_effects(mbeta, "treatment") # conditions = data.frame(size = 1)
conditional_effects(mbeta2, "treatment") # conditions = data.frame(size = 1)
conditional_effects(mbeta3, "treatment") # conditions = data.frame(size = 1)
conditional_effects(mbeta4, "treatment") # conditions = data.frame(size = 1)
conditional_effects(mbeta5, "treatment") # conditions = data.frame(size = 1)
conditional_effects(mbeta6, "treatment") # conditions = data.frame(size = 1)
Mean PPC
#q75 <- function(y) quantile(y, 0.75)
pp_check(mbeta, type='stat', stat='mean')
#+scale_x_continuous(breaks = seq(-0.5, 0.5, length.out = 7), labels = c("1", "5","10","15","20","25", "30"))#q75 <- function(y) quantile(y, 0.75)
pp_check(mbeta2, type='stat', stat='mean')
#+scale_x_continuous(breaks = seq(-0.5, 0.5, length.out = 7), labels = c("1", "5","10","15","20","25", "30"))#q75 <- function(y) quantile(y, 0.75)
pp_check(mbeta3, type='stat', stat='mean')
#+scale_x_continuous(breaks = seq(-0.5, 0.5, length.out = 7), labels = c("1", "5","10","15","20","25", "30"))#q75 <- function(y) quantile(y, 0.75)
pp_check(mbeta4, type='stat', stat='mean')
#+scale_x_continuous(breaks = seq(-0.5, 0.5, length.out = 7), labels = c("1", "5","10","15","20","25", "30"))#q75 <- function(y) quantile(y, 0.75)
pp_check(mbeta5, type='stat', stat='mean')
#+scale_x_continuous(breaks = seq(-0.5, 0.5, length.out = 7), labels = c("1", "5","10","15","20","25", "30"))#q75 <- function(y) quantile(y, 0.75)
pp_check(mbeta6, type='stat', stat='mean')
#+scale_x_continuous(breaks = seq(-0.5, 0.5, length.out = 7), labels = c("1", "5","10","15","20","25", "30"))bayesplot::pp_check(mbeta, type = "violin_grouped", group = "treatment")
bayesplot::pp_check(mbeta, type = "error_hist")
Plots
eval_period_ends = rq3 %>%
data_grid(
treatment,
eval_period_normalized = c(-0.467, .5) #c(-0.467, .5) # because we normalized trial to be from -0.5 to 0.5
) %>%
add_fitted_samples(mbeta, re_formula = NA, var = "mu", dpar = TRUE) %>%
ungroup() %>%
mutate(treatment = fct_rev(fct_relevel(treatment, vis_display_order))) %>%
mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))
#Conditional means (for "average" person on "average" scenario) on last trial:
eval_period_ends %>%
mutate(evalPeriod = if_else(eval_period_normalized==0.5,"30","1")) %>%
mutate(evalPeriod = factor(evalPeriod, levels = c("30","1"))) %>%
ggplot(aes(y = treatment, x = mu, group = evalPeriod, fill = evalPeriod)) +
geom_halfeyeh(fun.data = median_qih, fatten.point = 1.3) +
scale_fill_OkabeIto(alpha = 0.5) +
geom_vline(xintercept = 1, linetype = "dashed", color = "black") +
cowplot::theme_minimal_vgrid()
eval_period_ends2 = rq3 %>%
data_grid(
treatment,
eval_period_normalized = c(-0.467, .5) #c(-0.467, .5) # because we normalized trial to be from -0.5 to 0.5
) %>%
add_fitted_samples(mbeta2, re_formula = NA, var = "mu", dpar = TRUE) %>%
ungroup() %>%
mutate(treatment = fct_rev(fct_relevel(treatment, vis_display_order))) %>%
mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))
#Conditional means (for "average" person on "average" scenario) on last trial:
eval_period_ends2 %>%
mutate(evalPeriod = if_else(eval_period_normalized==0.5,"30","1")) %>%
mutate(evalPeriod = factor(evalPeriod, levels = c("30","1"))) %>%
ggplot(aes(y = treatment, x = mu, group = evalPeriod, fill = evalPeriod)) +
geom_halfeyeh(fun.data = median_qih, fatten.point = 1.3) +
scale_fill_OkabeIto(alpha = 0.5) +
geom_vline(xintercept = 1, linetype = "dashed", color = "black") +
cowplot::theme_minimal_vgrid()
Notice how when we remove interaction density seems to be significant. This is because we’re incorrectly applying the population’s evaluation curve.
eval_period_ends3 = rq3 %>%
data_grid(
treatment,
eval_period_normalized = c(-0.467, .5) #c(-0.467, .5) # because we normalized trial to be from -0.5 to 0.5
) %>%
add_fitted_samples(mbeta3, re_formula = NA, var = "mu", dpar = TRUE) %>%
ungroup() %>%
mutate(treatment = fct_rev(fct_relevel(treatment, vis_display_order))) %>%
mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))
#Conditional means (for "average" person on "average" scenario) on last trial:
eval_period_ends3 %>%
mutate(evalPeriod = if_else(eval_period_normalized==0.5,"30","1")) %>%
mutate(evalPeriod = factor(evalPeriod, levels = c("30","1"))) %>%
ggplot(aes(y = treatment, x = mu, group = evalPeriod, fill = evalPeriod)) +
geom_halfeyeh(fun.data = median_qih, fatten.point = 1.3) +
scale_fill_OkabeIto(alpha = 0.5) +
geom_vline(xintercept = 1, linetype = "dashed", color = "black") +
cowplot::theme_minimal_vgrid()
eval_period_ends4 = rq3 %>%
data_grid(
treatment,
eval_period_normalized = c(-0.467, .5) #c(-0.467, .5) # because we normalized trial to be from -0.5 to 0.5
) %>%
add_fitted_samples(mbeta4, re_formula = NA, var = "mu", dpar = TRUE) %>%
ungroup() %>%
mutate(treatment = fct_rev(fct_relevel(treatment, vis_display_order))) %>%
mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))
#Conditional means (for "average" person on "average" scenario) on last trial:
eval_period_ends4 %>%
mutate(evalPeriod = if_else(eval_period_normalized==0.5,"30","1")) %>%
mutate(evalPeriod = factor(evalPeriod, levels = c("30","1"))) %>%
ggplot(aes(y = treatment, x = mu, group = evalPeriod, fill = evalPeriod)) +
geom_halfeyeh(fun.data = median_qih, fatten.point = 1.3) +
scale_fill_OkabeIto(alpha = 0.5) +
geom_vline(xintercept = 1, linetype = "dashed", color = "black") +
cowplot::theme_minimal_vgrid()
eval_period_ends5 = rq3 %>%
data_grid(
treatment,
eval_period_normalized = c(-0.467, .5) #c(-0.467, .5) # because we normalized trial to be from -0.5 to 0.5
) %>%
add_fitted_samples(mbeta5, re_formula = NA, var = "mu", dpar = TRUE) %>%
ungroup() %>%
mutate(treatment = fct_rev(fct_relevel(treatment, vis_display_order))) %>%
mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))
#Conditional means (for "average" person on "average" scenario) on last trial:
eval_period_ends5 %>%
mutate(evalPeriod = if_else(eval_period_normalized==0.5,"30","1")) %>%
mutate(evalPeriod = factor(evalPeriod, levels = c("30","1"))) %>%
ggplot(aes(y = treatment, x = mu, group = evalPeriod, fill = evalPeriod)) +
geom_halfeyeh(fun.data = median_qih, fatten.point = 1.3) +
scale_fill_OkabeIto(alpha = 0.5) +
geom_vline(xintercept = 1, linetype = "dashed", color = "black") +
cowplot::theme_minimal_vgrid()
Again, without interaction we’re having issues.
eval_period_ends6 = rq3 %>%
data_grid(
treatment,
eval_period_normalized = c(-0.467, .5) #c(-0.467, .5) # because we normalized trial to be from -0.5 to 0.5
) %>%
add_fitted_samples(mbeta6, re_formula = NA, var = "mu", dpar = TRUE) %>%
ungroup() %>%
mutate(treatment = fct_rev(fct_relevel(treatment, vis_display_order))) %>%
mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))
#Conditional means (for "average" person on "average" scenario) on last trial:
eval_period_ends6 %>%
mutate(evalPeriod = if_else(eval_period_normalized==0.5,"30","1")) %>%
mutate(evalPeriod = factor(evalPeriod, levels = c("30","1"))) %>%
ggplot(aes(y = treatment, x = mu, group = evalPeriod, fill = evalPeriod)) +
geom_halfeyeh(fun.data = median_qih, fatten.point = 1.3) +
scale_fill_OkabeIto(alpha = 0.5) +
geom_vline(xintercept = 1, linetype = "dashed", color = "black") +
cowplot::theme_minimal_vgrid()
max_eval=30
mu_mean = rq3 %>%
mutate(
eval_period_normalized = ((evalPeriod - max_eval) / max_eval) + 0.5
) %>%
group_by(treatment, eval_period_normalized) %>%
summarise(mean=mean(dv))fit_lines6 = rq3 %>%
data_grid(
treatment,
eval_period_normalized = seq_range(eval_period_normalized, n = 20)
) %>%
add_fitted_samples(mbeta6, re_formula = NA, var = "mu", dpar = TRUE) %>%
ungroup() %>%
mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))
mu_mean6 = fit_lines6 %>%
group_by(treatment, eval_period_normalized) %>%
summarise(mu_mean=mean(mu))fit_lines = rq3 %>%
data_grid(
treatment,
eval_period_normalized = seq_range(eval_period_normalized, n = 20)
) %>%
add_fitted_samples(mbeta, re_formula = NA, var = "mu", dpar = TRUE) %>%
ungroup()
#mutate(vis = fct_relevel(vis, vis_display_order))As we see here, the model is not doing well. We need the interaction effect.
scale_fill_fit_lines = scale_fill_manual(
values = RColorBrewer::brewer.pal(4, "Greys")[-1], guide = guide_legend(reverse = TRUE)
)
fit_lines %>%
ggplot(aes(x = eval_period_normalized, y = mu)) +
stat_lineribbon(.prob = c(.95, .8, .5)) +
geom_hline(yintercept = seq(.7, 1, by=.05), color="gray75", alpha = 0.5) +
geom_line(aes(y = mean), data = mu_mean, color = "red", size = 1) +
facet_grid(. ~ forcats::fct_relevel(treatment, vis_display_order)) +
scale_fill_fit_lines 
#fit_line_plot_settingsThis is the actual model
scale_fill_fit_lines = scale_fill_manual(
values = RColorBrewer::brewer.pal(4, "Greys")[-1], guide = guide_legend(reverse = TRUE)
)
fit_lines %>%
ggplot(aes(x = eval_period_normalized, y = mu)) +
stat_lineribbon(.prob = c(.95, .8, .5)) +
geom_hline(yintercept = seq(.7, 1, by=.05), color="gray75", alpha = 0.5) +
geom_line(aes(y = mean), data = mu_mean, color = "red", size = 1) +
facet_grid(. ~ forcats::fct_relevel(treatment, vis_display_order)) +
scale_fill_fit_lines
#fit_line_plot_settings